{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cot import Collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading aqua...\n",
      "Loading asdiv...\n",
      "Loading commonsense_qa...\n",
      "Loading entailment_bank...\n",
      "Loading gsm8k...\n",
      "Loading mawps...\n",
      "Loading med_qa...\n",
      "Loading medmc_qa...\n",
      "Loading open_book_qa...\n",
      "Loading pubmed_qa...\n",
      "Loading qed...\n",
      "Loading strategy_qa...\n",
      "Loading svamp...\n",
      "Loading worldtree...\n"
     ]
    }
   ],
   "source": [
    "collection_all_lievin = Collection(\"all\", load_pregenerated_cots=[\"lievin\", \"lievin_100\"], verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name            |   Train | Valid   | Test   |\n",
       "|-----------------|---------|---------|--------|\n",
       "| aqua            |   97467 | 254     | 254    |\n",
       "| asdiv           |    1218 | -       | -      |\n",
       "| commonsense_qa  |    9741 | 1221    | 1140   |\n",
       "| entailment_bank |    1313 | 187     | 340    |\n",
       "| gsm8k           |    7473 | -       | 1319   |\n",
       "| mawps           |    1921 | -       | -      |\n",
       "| med_qa          |   10178 | 1272    | 1273   |\n",
       "| medmc_qa        |  182822 | 4183    | 6150   |\n",
       "| open_book_qa    |    4957 | 500     | 500    |\n",
       "| pubmed_qa       |     500 | -       | 500    |\n",
       "| qed             |    5154 | 1021    | -      |\n",
       "| strategy_qa     |    2290 | -       | 490    |\n",
       "| svamp           |    1000 | -       | -      |\n",
       "| worldtree       |    2207 | 496     | 1664   |\n",
       "\n",
       "Not loaded: []"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "collection_all_lievin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll_ts_huggingface = Collection.from_json(\"./thoughtsource_data/thoughtsource_100_huggingface.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll_ts_openai_cohere = Collection.from_json(\"./thoughtsource_data/thoughtsource_100_openai_cohere.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll_ts_huggingface_openai_cohere = coll_ts_huggingface.merge(coll_ts_openai_cohere)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "collection_paper1 = collection_all_lievin.merge(coll_ts_huggingface_openai_cohere)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "collection_paper1.dump(\"collection_paper1.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cot.stats import display_stats_tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": [
       "\u001b[?25l"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea589ebfc52840dd9dd4b210b0ab28df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[?25h"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Train</th>\n",
       "      <th>Valid</th>\n",
       "      <th>Test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aqua</td>\n",
       "      <td>97467</td>\n",
       "      <td>254</td>\n",
       "      <td>254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>asdiv</td>\n",
       "      <td>1218</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>commonsense_qa</td>\n",
       "      <td>9741</td>\n",
       "      <td>1221</td>\n",
       "      <td>1140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>entailment_bank</td>\n",
       "      <td>1313</td>\n",
       "      <td>187</td>\n",
       "      <td>340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gsm8k</td>\n",
       "      <td>7473</td>\n",
       "      <td>-</td>\n",
       "      <td>1319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>mawps</td>\n",
       "      <td>1921</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>med_qa</td>\n",
       "      <td>10178</td>\n",
       "      <td>1272</td>\n",
       "      <td>1273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>medmc_qa</td>\n",
       "      <td>182822</td>\n",
       "      <td>4183</td>\n",
       "      <td>6150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>open_book_qa</td>\n",
       "      <td>4957</td>\n",
       "      <td>500</td>\n",
       "      <td>500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>pubmed_qa</td>\n",
       "      <td>500</td>\n",
       "      <td>-</td>\n",
       "      <td>500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>qed</td>\n",
       "      <td>5154</td>\n",
       "      <td>1021</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>strategy_qa</td>\n",
       "      <td>2290</td>\n",
       "      <td>-</td>\n",
       "      <td>490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>svamp</td>\n",
       "      <td>1000</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>worldtree</td>\n",
       "      <td>2207</td>\n",
       "      <td>496</td>\n",
       "      <td>1664</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Name   Train Valid  Test\n",
       "0              aqua   97467   254   254\n",
       "1             asdiv    1218     -     -\n",
       "2    commonsense_qa    9741  1221  1140\n",
       "3   entailment_bank    1313   187   340\n",
       "4             gsm8k    7473     -  1319\n",
       "5             mawps    1921     -     -\n",
       "6            med_qa   10178  1272  1273\n",
       "7          medmc_qa  182822  4183  6150\n",
       "8      open_book_qa    4957   500   500\n",
       "9         pubmed_qa     500     -   500\n",
       "10              qed    5154  1021     -\n",
       "11      strategy_qa    2290     -   490\n",
       "12            svamp    1000     -     -\n",
       "13        worldtree    2207   496  1664"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>question</th>\n",
       "      <th>choices</th>\n",
       "      <th>cot</th>\n",
       "      <th>generated_cot</th>\n",
       "      <th>question_with_generated_cot</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aqua</td>\n",
       "      <td>97975</td>\n",
       "      <td>97975</td>\n",
       "      <td>97975</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>97975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>asdiv</td>\n",
       "      <td>1218</td>\n",
       "      <td>0</td>\n",
       "      <td>1218</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>commonsense_qa</td>\n",
       "      <td>12102</td>\n",
       "      <td>12102</td>\n",
       "      <td>10962</td>\n",
       "      <td>600</td>\n",
       "      <td>100</td>\n",
       "      <td>12102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>entailment_bank</td>\n",
       "      <td>1840</td>\n",
       "      <td>0</td>\n",
       "      <td>1840</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gsm8k</td>\n",
       "      <td>8792</td>\n",
       "      <td>0</td>\n",
       "      <td>8792</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>mawps</td>\n",
       "      <td>1921</td>\n",
       "      <td>0</td>\n",
       "      <td>1921</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>med_qa</td>\n",
       "      <td>12723</td>\n",
       "      <td>12723</td>\n",
       "      <td>0</td>\n",
       "      <td>134260</td>\n",
       "      <td>1273</td>\n",
       "      <td>12723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>medmc_qa</td>\n",
       "      <td>193155</td>\n",
       "      <td>193155</td>\n",
       "      <td>161558</td>\n",
       "      <td>105587</td>\n",
       "      <td>1000</td>\n",
       "      <td>193155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>open_book_qa</td>\n",
       "      <td>5957</td>\n",
       "      <td>5957</td>\n",
       "      <td>5957</td>\n",
       "      <td>600</td>\n",
       "      <td>100</td>\n",
       "      <td>5957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>pubmed_qa</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>2500</td>\n",
       "      <td>500</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>qed</td>\n",
       "      <td>6175</td>\n",
       "      <td>0</td>\n",
       "      <td>6175</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>strategy_qa</td>\n",
       "      <td>2780</td>\n",
       "      <td>0</td>\n",
       "      <td>2290</td>\n",
       "      <td>600</td>\n",
       "      <td>100</td>\n",
       "      <td>2780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>svamp</td>\n",
       "      <td>1000</td>\n",
       "      <td>0</td>\n",
       "      <td>1000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>worldtree</td>\n",
       "      <td>4367</td>\n",
       "      <td>4367</td>\n",
       "      <td>4365</td>\n",
       "      <td>601</td>\n",
       "      <td>100</td>\n",
       "      <td>4367</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            dataset  question  choices     cot  generated_cot  \\\n",
       "0              aqua     97975    97975   97975              0   \n",
       "1             asdiv      1218        0    1218              0   \n",
       "2    commonsense_qa     12102    12102   10962            600   \n",
       "3   entailment_bank      1840        0    1840              0   \n",
       "4             gsm8k      8792        0    8792              0   \n",
       "5             mawps      1921        0    1921              0   \n",
       "6            med_qa     12723    12723       0         134260   \n",
       "7          medmc_qa    193155   193155  161558         105587   \n",
       "8      open_book_qa      5957     5957    5957            600   \n",
       "9         pubmed_qa      1000     1000    1000           2500   \n",
       "10              qed      6175        0    6175              0   \n",
       "11      strategy_qa      2780        0    2290            600   \n",
       "12            svamp      1000        0    1000              0   \n",
       "13        worldtree      4367     4367    4365            601   \n",
       "\n",
       "    question_with_generated_cot  answer  \n",
       "0                             0   97975  \n",
       "1                             0    1218  \n",
       "2                           100   12102  \n",
       "3                             0    1840  \n",
       "4                             0    8792  \n",
       "5                             0    1921  \n",
       "6                          1273   12723  \n",
       "7                          1000  193155  \n",
       "8                           100    5957  \n",
       "9                           500    1000  \n",
       "10                            0    6175  \n",
       "11                          100    2780  \n",
       "12                            0    1000  \n",
       "13                          100    4367  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>number samples</th>\n",
       "      <th>datasets</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>multiplechoice</td>\n",
       "      <td>327279</td>\n",
       "      <td>{aqua, pubmed_qa, commonsense_qa, med_qa, medm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>number</td>\n",
       "      <td>12931</td>\n",
       "      <td>{svamp, gsm8k, mawps, asdiv}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>text</td>\n",
       "      <td>1840</td>\n",
       "      <td>{entailment_bank}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>collection</td>\n",
       "      <td>6175</td>\n",
       "      <td>{qed}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bool</td>\n",
       "      <td>2780</td>\n",
       "      <td>{strategy_qa}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             type  number samples  \\\n",
       "0  multiplechoice          327279   \n",
       "1          number           12931   \n",
       "2            text            1840   \n",
       "3      collection            6175   \n",
       "4            bool            2780   \n",
       "\n",
       "                                            datasets  \n",
       "0  {aqua, pubmed_qa, commonsense_qa, med_qa, medm...  \n",
       "1                       {svamp, gsm8k, mawps, asdiv}  \n",
       "2                                  {entailment_bank}  \n",
       "3                                              {qed}  \n",
       "4                                      {strategy_qa}  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "table_num_examples, table_nan, table_types = display_stats_tables(collection_paper1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "ccbfa654f25866afe66a1c016d0b518a994fbe20a93c2d8b432dbe114020e2d2"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
